
rm(list = ls())

library('data.table')
library('stargazer')
library('ggplot2')
library('reldist')
library('matlab')


#########################################################################################################################

#################################
# Prepare Dataset for Product 1 #
#################################

# Import and Clean Data: 1-1000 

i_k = 1
data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
data_1 = read.table(data_name, header =TRUE, sep=",")
data_1 <- as.data.table(data_1)

# Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
data_1 <- data_1[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]

data_1 <- data_1[ , product := i_k]   

I_K = 1000
for (i_k in 2:I_K) {
  
  print(i_k)  
  
  # Import and Clean Data  
  data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
  data_temp = read.table(data_name, header =TRUE, sep=",")
  data_temp <- as.data.table(data_temp)
  
  # Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
  data_temp <- data_temp[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]
  
  data_temp <- data_temp[ , product := i_k]  
  
  data_1 = rbind(data_1, data_temp)
  
}  



# Import and Clean Data: 1001-2000 

i_k = 1001
data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
data_2 = read.table(data_name, header =TRUE, sep=",")
data_2 <- as.data.table(data_2)

# Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
data_2 <- data_2[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]

data_2 <- data_2[ , product := i_k]   

I_K = 2000
for (i_k in 1002:I_K) {
  
  print(i_k)  
  
  # Import and Clean Data  
  data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
  data_temp = read.table(data_name, header =TRUE, sep=",")
  data_temp <- as.data.table(data_temp)
  
  # Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
  data_temp <- data_temp[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]
  
  data_temp <- data_temp[ , product := i_k]  
  
  data_2 = rbind(data_2, data_temp)
  
}  


# Import and Clean Data: 2001-3000 

i_k = 2001
data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
data_3 = read.table(data_name, header =TRUE, sep=",")
data_3 <- as.data.table(data_3)

# Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
data_3 <- data_3[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]

data_3 <- data_3[ , product := i_k]   

I_K = 3000
for (i_k in 2002:I_K) {
  
  print(i_k)  
  
  # Import and Clean Data  
  data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
  data_temp = read.table(data_name, header =TRUE, sep=",")
  data_temp <- as.data.table(data_temp)
  
  # Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
  data_temp <- data_temp[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]
  
  data_temp <- data_temp[ , product := i_k]  
  
  data_3 = rbind(data_3, data_temp)
  
}  


# Import and Clean Data: 3001-4000 

i_k = 3001
data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
data_4 = read.table(data_name, header =TRUE, sep=",")
data_4 <- as.data.table(data_4)

# Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
data_4 <- data_4[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]

data_4 <- data_4[ , product := i_k]   

I_K = 4092
for (i_k in 3002:I_K) {
  
  print(i_k)  
  
  # Import and Clean Data  
  data_name <- sprintf("price_index/product_specific_gini/diff_full_adj_table_%s_k.csv", i_k)
  data_temp = read.table(data_name, header =TRUE, sep=",")
  data_temp <- as.data.table(data_temp)
  
  # Create index that is 1 if |beta| < 0.6 or if the Home country's mc could not be inferred (demand not sufficiently elastic)
  data_temp <- data_temp[ , index_mc := (any(abs(beta_price_80) < 0.6) | any(abs(beta_price_20) < 0.6) | any(Declarant == Partner & mc_flag == -2)), by = c('Year', 'Quarter', 'Declarant')]
  
  data_temp <- data_temp[ , product := i_k]  
  
  data_4 = rbind(data_4, data_temp)
  
}  


data <- rbind(data_1, data_2, data_3, data_4)

save(file = 'data_PI_full_adj_gini.RData', data)


